﻿using ConsoleTest.Model;
using ConsoleTest.Tools;
using HtmlAgilityPack;
using System;
using System.Collections.Generic;
using System.Configuration;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;

namespace ConsoleTest
{
    class Program
    {
       
        static void Main(string[] args)
        {
            Service.Service service = new Service.Service();
            bool isReadFile = Convert.ToBoolean(ConfigurationManager.AppSettings["IsReadFile"]);
            GetHtml gh = new GetHtml();
            ExcelRed ed = new ExcelRed();
            Console.WriteLine("开始采集" + "开始时间：" + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
            StringBuilder sbTitle = new StringBuilder();

          
            if (isReadFile)
            {
                var titles = ed.GetTitle();
                //  StringBuilder sb = new StringBuilder();
                var listWord = new List<KeyWord>();

                for (int i = 0; i < titles.Count; i++)
                {
                    listWord.Add(new KeyWord { Id = i + 1, Title = titles[i] });
                    // sb.Append(string.Format("insert into KeyWord(Title)values('{0}');", titles[i]));
                }
                SqlHelper.BulkInsert("keyword", listWord);
            }

            var list = SqlHelper.GetEntitys<KeyWord>("select * from keyword");

            service.Method360(list);

            //把关键词以500分一组 此做法是方便采集一部分保存一部分，
            //var listGroup = new List<List<KeyWord>>();
            //var j = 10;
            //for (var i = 0; i < list.Count; i += 10)
            //{
            //    var cList = list.Take(j).Skip(i).ToList();
            //    j += 10;
            //    listGroup.Add(cList);
            //}

            //foreach (var group in listGroup)
            //{
            int count = 0;
            foreach (var keys in list)
            {
                //第一步通过关键词爬去百度下拉推荐标题
                var url = string.Format("https://sp0.baidu.com/5a1Fazu8AA54nxGko9WTAnF6hhy/su?wd={0}", keys.Title);
                //关键词对应推荐的标题
                var html = gh.GetHtmlWithUtf(url, Encoding.Default);
                var roots = html.Substring(17, html.Length - 19).ToObject<Root>();
                Console.WriteLine("关键词-----------'" + roots.q + "'---------\r\n");
                foreach (var item in roots.s)
                {
                    sbTitle.Append("insert into TitleWord(title,IsDropdown,Platform,parentid) values('" + item + "',1,'百度'," + keys.Id + ");");
                }
                //第二部通过关键词爬去百度底部相关搜索推荐标题
                var bottomUrl = string.Format("http://www.baidu.com/s?wd={0}&pn=0&oq={1}", keys.Title, keys.Title);
                var bottomhtml = gh.GetHtmlWithUtf(bottomUrl, Encoding.UTF8);
                var hd = new HtmlDocument();
                hd.LoadHtml(bottomhtml);
                HtmlNode root = hd.DocumentNode;
                //获取 companyname
                HtmlNodeCollection nodeCompanyName = root.SelectNodes("//div[@id='rs']/table/tr/th");
                if (nodeCompanyName != null)
                {
                    foreach (var m in nodeCompanyName)
                    {
                        var aTag = m.InnerHtml;//a标签获取

                        Regex reg = new Regex(@"(?is)<a[^>]+?href=(['""]?)(?<url>[^'""\s>]+)\1[^>]*>(?<text>(?:(?!</?a\b).)*)</a>");
                        MatchCollection mc = reg.Matches(aTag);
                        foreach (Match item in mc)
                        {
                            var title = item.Groups["text"].Value;//获取到title

                            if (!roots.s.Contains(title))
                            {
                                // roots.s.Add(title);
                                sbTitle.Append("insert into TitleWord(title,IsDropdown,Platform,parentid) values('" + title + "',0,'百度'," + keys.Id + ");");
                            }
                        }
                    }
                }
                Console.WriteLine("关键词采集完毕" + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss") + "第" + keys.Id + "条\r\n");

                count++;
                if (count % 100 == 0 || count == list.Count)
                {
                    string sqlTitle = sbTitle.ToString();
                    var countTitle = SqlHelper.ExecuteNonQuery(sqlTitle);
                    sqlTitle.Remove(0, sqlTitle.Length);
                }
            }
            // }

            //sb.Append("insert into KeyWord(Title)values('"+titles[i]+"');");

            //   Console.WriteLine("第二步开始360采集");
            //第二步 ，360搜索
            //  var url_360 = "http://wenda.so.com/search/?q=百里香酚";
            //  var html_360 = gh.GetHtmlWithUtf(url_360, Encoding.Default);

            Console.ReadKey();
        }
    }
}
